package engine

import (
	"fmt"
	"cn.hughie/learngo/crawler1/fetcher"
)

func Go(requests ...Request) {
	q := []Request{}
	q = append(q, requests...)
	for len(q) > 0 {
		request := q[0]
		q = q[1:]
		contents, e := fetcher.Fetch(request.Url)
		if e != nil {
			continue
		}
		result := request.ParseFunc(contents)
		for _, item := range result.Items {
			fmt.Printf("Got persist %v \n", item)
		}
		q = append(q, result.Requests...)
	}
}
